{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bayesian Optimization\n",
    "https://github.com/fmfn/BayesianOptimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')  # 不打印警告\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基本用法"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 安装\n",
    "pip install bayesian-optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bayes_opt import BayesianOptimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 目标函数\n",
    "$ f(x, y) = 1 - x^2 - (y - 1)^2 $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = lambda x, y: 1 - x ** 2 - (y - 1) ** 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 参数范围"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pbounds = {'x': (-1, 4), 'y': (-3, 3)}  # Bounded region of parameter space"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 创建优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = BayesianOptimization(f=f, pbounds=pbounds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 优化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |     x     |     y     |\n",
      "-------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-12.42   \u001b[0m | \u001b[0m 3.473   \u001b[0m | \u001b[0m 2.166   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-6.326   \u001b[0m | \u001b[95m 2.552   \u001b[0m | \u001b[95m 0.09898 \u001b[0m |\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m-3.167   \u001b[0m | \u001b[95m 2.032   \u001b[0m | \u001b[95m 0.801   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-2.754   \u001b[0m | \u001b[95m 1.9     \u001b[0m | \u001b[95m 0.6212  \u001b[0m |\n",
      "| \u001b[95m 5       \u001b[0m | \u001b[95m-1.993   \u001b[0m | \u001b[95m 1.717   \u001b[0m | \u001b[95m 0.7872  \u001b[0m |\n",
      "| \u001b[95m 6       \u001b[0m | \u001b[95m-1.289   \u001b[0m | \u001b[95m 1.474   \u001b[0m | \u001b[95m 0.6614  \u001b[0m |\n",
      "| \u001b[95m 7       \u001b[0m | \u001b[95m-0.4572  \u001b[0m | \u001b[95m 1.198   \u001b[0m | \u001b[95m 0.8536  \u001b[0m |\n",
      "| \u001b[95m 8       \u001b[0m | \u001b[95m 0.2396  \u001b[0m | \u001b[95m 0.8247  \u001b[0m | \u001b[95m 0.7167  \u001b[0m |\n",
      "| \u001b[95m 9       \u001b[0m | \u001b[95m 0.7526  \u001b[0m | \u001b[95m 0.4955  \u001b[0m | \u001b[95m 1.044   \u001b[0m |\n",
      "| \u001b[95m 10      \u001b[0m | \u001b[95m 0.9504  \u001b[0m | \u001b[95m 0.06238 \u001b[0m | \u001b[95m 0.7863  \u001b[0m |\n",
      "=================================================\n"
     ]
    }
   ],
   "source": [
    "optimizer.maximize(init_points=3, n_iter=7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'target': -12.424507752814025,\n",
       "  'params': {'x': 3.473381983379655, 'y': 2.1662441212489085}},\n",
       " {'target': -6.325879627577446,\n",
       "  'params': {'x': 2.5522609732566512, 'y': 0.09897638656444885}},\n",
       " {'target': -3.1673181574650227,\n",
       "  'params': {'x': 2.031679275354415, 'y': 0.8010088455222633}},\n",
       " {'target': -2.753730951498059,\n",
       "  'params': {'x': 1.9000679292636926, 'y': 0.6212219440336568}},\n",
       " {'target': -1.9927674788639687,\n",
       "  'params': {'x': 1.7168184591075526, 'y': 0.7871576702543933}},\n",
       " {'target': -1.2885134239882254,\n",
       "  'params': {'x': 1.474408254666627, 'y': 0.6614239781686598}},\n",
       " {'target': -0.4571749329020343,\n",
       "  'params': {'x': 1.1982192065174164, 'y': 0.8535566115018938}},\n",
       " {'target': 0.239643660045436,\n",
       "  'params': {'x': 0.8246929302447152, 'y': 0.7167370289661049}},\n",
       " {'target': 0.7526118466639338,\n",
       "  'params': {'x': 0.4954534162613094, 'y': 1.0437500360121434}},\n",
       " {'target': 0.9504485354889007,\n",
       "  'params': {'x': 0.062381868231868344, 'y': 0.7863180704247524}}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'target': 0.9504485354889007,\n",
       " 'params': {'x': 0.062381868231868344, 'y': 0.7863180704247524}}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-12.42450775,  -6.32587963,  -3.16731816,  -2.75373095,\n",
       "        -1.99276748,  -1.28851342,  -0.45717493,   0.23964366,\n",
       "         0.75261185,   0.95044854])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.space.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "optimizer.space.target.argmax()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sklearn示例\n",
    "https://github.com/fmfn/BayesianOptimization/blob/master/examples/sklearn_example.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | max_fe... | min_sa... | n_esti... |\n",
      "-------------------------------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-0.3773  \u001b[0m | \u001b[0m 0.2722  \u001b[0m | \u001b[0m 16.31   \u001b[0m | \u001b[0m 115.1   \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m-0.3654  \u001b[0m | \u001b[95m 0.806   \u001b[0m | \u001b[95m 19.94   \u001b[0m | \u001b[95m 75.42   \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m-0.3756  \u001b[0m | \u001b[0m 0.3485  \u001b[0m | \u001b[0m 20.44   \u001b[0m | \u001b[0m 240.0   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m-0.354   \u001b[0m | \u001b[95m 0.8875  \u001b[0m | \u001b[95m 10.23   \u001b[0m | \u001b[95m 130.2   \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-0.3648  \u001b[0m | \u001b[0m 0.7144  \u001b[0m | \u001b[0m 18.39   \u001b[0m | \u001b[0m 98.86   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-0.582   \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 10.0    \u001b[0m |\n",
      "| \u001b[95m 7       \u001b[0m | \u001b[95m-0.3516  \u001b[0m | \u001b[95m 0.999   \u001b[0m | \u001b[95m 2.0     \u001b[0m | \u001b[95m 250.0   \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m-0.3518  \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 197.9   \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m-0.4275  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 80.14   \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m-0.3753  \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 25.0    \u001b[0m | \u001b[0m 166.8   \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m-0.3535  \u001b[0m | \u001b[0m 0.9979  \u001b[0m | \u001b[0m 2.031   \u001b[0m | \u001b[0m 160.1   \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-0.4743  \u001b[0m | \u001b[0m 0.999   \u001b[0m | \u001b[0m 25.0    \u001b[0m | \u001b[0m 10.0    \u001b[0m |\n",
      "| \u001b[95m 13      \u001b[0m | \u001b[95m-0.3506  \u001b[0m | \u001b[95m 0.9604  \u001b[0m | \u001b[95m 2.071   \u001b[0m | \u001b[95m 225.8   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m-0.3733  \u001b[0m | \u001b[0m 0.995   \u001b[0m | \u001b[0m 24.13   \u001b[0m | \u001b[0m 205.8   \u001b[0m |\n",
      "| \u001b[0m 15      \u001b[0m | \u001b[0m-0.3521  \u001b[0m | \u001b[0m 0.974   \u001b[0m | \u001b[0m 2.054   \u001b[0m | \u001b[0m 135.6   \u001b[0m |\n",
      "=============================================================\n",
      "Final result: {'target': -0.35056171127031877, 'params': {'max_features': 0.9603785406423503, 'min_samples_split': 2.0713022103577927, 'n_estimators': 225.84014817033898}}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.ensemble import RandomForestClassifier as RFC\n",
    "from bayes_opt import BayesianOptimization\n",
    "\n",
    "\n",
    "def get_data():\n",
    "    data, targets = make_classification(\n",
    "        n_samples=1000,\n",
    "        n_features=45,\n",
    "        n_informative=12,\n",
    "        n_redundant=7,\n",
    "        random_state=134985745,\n",
    "    )\n",
    "    return data, targets\n",
    "\n",
    "\n",
    "def rfc_cv(n_estimators, min_samples_split, max_features, data, targets):\n",
    "    estimator = RFC(\n",
    "        n_estimators=n_estimators,\n",
    "        min_samples_split=min_samples_split,\n",
    "        max_features=max_features,\n",
    "        random_state=2\n",
    "    )\n",
    "    cval = cross_val_score(estimator, data, targets, scoring='neg_log_loss', cv=4)\n",
    "    return cval.mean()\n",
    "\n",
    "\n",
    "def optimize_rfc(data, targets):\n",
    "    def rfc_crossval(n_estimators, min_samples_split, max_features):\n",
    "        return rfc_cv(\n",
    "            n_estimators=int(n_estimators),\n",
    "            min_samples_split=int(min_samples_split),\n",
    "            max_features=max(min(max_features, 0.999), 1e-3),\n",
    "            data=data,\n",
    "            targets=targets,\n",
    "        )\n",
    "\n",
    "    optimizer = BayesianOptimization(\n",
    "        f=rfc_crossval,\n",
    "        pbounds={\n",
    "            \"n_estimators\": (10, 250),\n",
    "            \"min_samples_split\": (2, 25),\n",
    "            \"max_features\": (0.1, 0.999),\n",
    "        },\n",
    "        random_state=1234,\n",
    "    )\n",
    "    optimizer.maximize(n_iter=10)\n",
    "\n",
    "    print(\"Final result:\", optimizer.max)\n",
    "\n",
    "\n",
    "data, targets = get_data()\n",
    "optimize_rfc(data, targets)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 返回最优已训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   |     C     |\n",
      "-------------------------------------\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m 0.7675  \u001b[0m | \u001b[0m-2.31    \u001b[0m |\n",
      "| \u001b[95m 2       \u001b[0m | \u001b[95m 0.79    \u001b[0m | \u001b[95m 1.51    \u001b[0m |\n",
      "| \u001b[0m 3       \u001b[0m | \u001b[0m 0.7875  \u001b[0m | \u001b[0m 1.847   \u001b[0m |\n",
      "| \u001b[95m 4       \u001b[0m | \u001b[95m 0.8     \u001b[0m | \u001b[95m-2.422   \u001b[0m |\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m 0.79    \u001b[0m | \u001b[0m 1.721   \u001b[0m |\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m 0.525   \u001b[0m | \u001b[0m-3.0     \u001b[0m |\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m 0.7925  \u001b[0m | \u001b[0m-0.2345  \u001b[0m |\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m 0.795   \u001b[0m | \u001b[0m 0.4802  \u001b[0m |\n",
      "| \u001b[0m 9       \u001b[0m | \u001b[0m 0.795   \u001b[0m | \u001b[0m-1.022   \u001b[0m |\n",
      "| \u001b[0m 10      \u001b[0m | \u001b[0m 0.795   \u001b[0m | \u001b[0m-0.6551  \u001b[0m |\n",
      "| \u001b[0m 11      \u001b[0m | \u001b[0m 0.795   \u001b[0m | \u001b[0m 0.9339  \u001b[0m |\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m 0.7925  \u001b[0m | \u001b[0m 0.1263  \u001b[0m |\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m 0.7975  \u001b[0m | \u001b[0m-1.419   \u001b[0m |\n",
      "| \u001b[0m 14      \u001b[0m | \u001b[0m 0.66    \u001b[0m | \u001b[0m-2.59    \u001b[0m |\n",
      "| \u001b[95m 15      \u001b[0m | \u001b[95m 0.8025  \u001b[0m | \u001b[95m-1.733   \u001b[0m |\n",
      "=====================================\n",
      "Best result: {'target': 0.8025, 'params': {'C': -1.7328121458932235}}\n"
     ]
    }
   ],
   "source": [
    "from warnings import filterwarnings\n",
    "filterwarnings('ignore')  # 不打印警告\n",
    "from sklearn.datasets import make_circles\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVC\n",
    "from bayes_opt import BayesianOptimization\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "\n",
    "\n",
    "ls_of_clf = []\n",
    "\n",
    "\n",
    "def load_data():\n",
    "    X, y = make_circles(n_samples=5000, noise=.3, factor=.4)\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.2)\n",
    "    X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=.1)\n",
    "    return X_train, y_train, X_valid, y_valid, X_test, y_test\n",
    "\n",
    "\n",
    "def score(X_train, y_train, X_valid, y_valid, C):\n",
    "    C = 10 ** C  # 幂\n",
    "    clf = SVC(C).fit(X_train, y_train)\n",
    "    ls_of_clf.append(clf)\n",
    "    return clf.score(X_valid, y_valid)\n",
    "\n",
    "\n",
    "def best_clf(X_train, y_train, X_valid, y_valid):\n",
    "    f = lambda C: score(X_train, y_train, X_valid, y_valid, C)\n",
    "    pbounds = {'C': (-3, 2)}  # 取对数使均匀采样【不用(1e-3, 1e2)】\n",
    "    optimizer = BayesianOptimization(f, pbounds)\n",
    "    optimizer.maximize(n_iter=10)\n",
    "    print('Best result:', optimizer.max)\n",
    "    idx = optimizer.space.target.argmax()\n",
    "    return ls_of_clf[idx]\n",
    "\n",
    "\n",
    "X_train, y_train, X_valid, y_valid, X_test, y_test = load_data()\n",
    "clf = best_clf(X_train, y_train, X_valid, y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8191666666666667\n",
      "0.8025\n",
      "0.841\n"
     ]
    }
   ],
   "source": [
    "print(clf.score(X_train, y_train))\n",
    "print(clf.score(X_valid, y_valid))\n",
    "print(clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8197222222222222\n",
      "0.795\n",
      "0.832\n"
     ]
    }
   ],
   "source": [
    "clf = SVC().fit(X_train, y_train)\n",
    "print(clf.score(X_train, y_train))\n",
    "print(clf.score(X_valid, y_valid))\n",
    "print(clf.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8185\n",
      "\n",
      "0.835\n"
     ]
    }
   ],
   "source": [
    "X_train = np.concatenate((X_train, X_valid), axis=0)\n",
    "y_train = np.concatenate((y_train, y_valid), axis=0)\n",
    "clf = SVC().fit(X_train, y_train)\n",
    "print(clf.score(X_train, y_train))\n",
    "print()\n",
    "print(clf.score(X_test, y_test))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
